/*
    Copyright 2006-2011 Patrik Jonsson, sunrise@familjenjonsson.org

    This file is part of Sunrise.

    Sunrise is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.

    Sunrise is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with Sunrise.  If not, see <http://www.gnu.org/licenses/>.

*/

/// \file
/// Contains a general function for shooting rays with multiple threads.

// $Id$

#ifndef __shoot__
#define __shoot__

#include "config.h"
#include <iostream> 
#include "counter.h"
#include "boost/thread/thread.hpp"
#include "boost/shared_ptr.hpp"
#include "hpm.h"
#include <vector> 
#include "xfer.h"
#include "terminator.h"
#include "threadlocal.h"

namespace mcrx {
  template<typename, typename> class shooter_thread;
  template<typename> class randomized_bias_shooter;
  template<typename> class scatter_shooter;
  template<typename> class nonscatter_shooter;

  template<typename T_xfer, typename T_shooter>
  bool
  shoot (T_xfer& xx,
	 T_shooter s, 
	 const terminator& t,
	 std::vector<T_rng::T_state>& rng_states,
	 long n_rays_desired,
	 long& n_rays,
	 int n_threads,
	 bool bind_threads,
	 int bank_size,
	 int hpm_no,
	 std::string hpm_string);
}


  
/** Functor class: shoot for the supplied xfer object. */
template<typename biaser_type>
class mcrx::scatter_shooter {
public:
  typedef biaser_type T_biaser;
private:
  T_biaser b;
public:
  scatter_shooter (T_biaser bb): b (bb) {};
  template<typename T_xfer> void operator() (T_xfer& x) const {
    x.shoot( b);}
};

/** Functor class: shoot_isotropic for the supplied xfer object. */
template<typename biaser_type>
class mcrx::nonscatter_shooter {
public:
  typedef biaser_type T_biaser;
private:
  T_biaser b;
public:
  nonscatter_shooter () {};
  nonscatter_shooter (T_biaser bb): b (bb) {};
  template<typename T_xfer> void operator() (T_xfer& x) const {
    x.shoot_isotropic (b);}
};


/** Functor class: calls shoot for the supplied xfer object, but with
    a biaser which is randomized between the specified limits. */
template<typename T_shooter>
class mcrx::randomized_bias_shooter {
  int minref_;
  int maxref_;
  mutable int i_;
  mutable int n_;
public:
  typedef typename T_shooter::T_biaser T_biaser;

  /** Constructs a shooter which randomly draws the reference
      wavelengths for its biaser object between minref and maxref.  */
  randomized_bias_shooter(int minref, int maxref) :
    minref_(minref), maxref_(maxref), n_(0) {};

  template<typename T_xfer> void operator() (T_xfer& x) const {
    // we only change reference wavelength every 10 times to avoid
    // recalculating weights all the time.
    if (n_ == 0) {
      i_ = int((maxref_-minref_+1) *x.rng().rnd()) + minref_;
      DEBUG(2,cout << "Changing reference wavelength to " << i_ << endl;);
      n_ = 10;
    }
    T_biaser b(i_);
    T_shooter s(b);
    s(x);
    --n_;
  };
};

const int cache_line_size = 128;

/** Thread object, executes the shooting loop in the shoot() function.
    The shooter_thread objects operator () executes in the thread,
    which is started by the shoot () function.  The object contains
    its own xfer object to do the radiative transfer and all
    additional necessary information. */
template<typename T_xfer, typename T_shooter>  
class mcrx::shooter_thread {
public:
  char padding [cache_line_size]; //!< To ensure threads don't share cache line

  /** Flag used to ensure that we've made a local copy of the object
      before the thread starts executing. */
  bool execute_;
  bool bind_threads_;

  int thread_number_;
  int bank_size_;

  T_xfer x; //!< Xfer object used by this thread
  /// Shooter functor knows how to tell the xfer object how to shoot a ray.
  const T_shooter s;
  /// Terminator functor knows how to check whether shooting should be
  /// terminated.

  const terminator& t;
  /// Reference to the state of the random number generator.
  T_rng::T_state& rng_state;
  /// Desired number of rays to be shot by THIS THREAD.
  long n_rays_desired;
  /// Reference to the ray counter, common for all threads.
  volatile long& n_rays;
  int hpm_no;
  std::string hpm_string;
  counter& c;
  /// Reference to the mutex protecting the ray counter.
  boost::mutex& n_rays_mutex;
public:
  shooter_thread(int n, T_xfer& xxx,
		 const T_shooter& ss, const terminator& tt,
		 T_rng::T_state& st, long nrd, long& nr, bool bt, int bs,
		 int hno,
		 const std::string& hstr, counter& cc, boost::mutex& nrm):
    execute_(false), thread_number_ (n), x (xxx), s (ss), t (tt), rng_state (st),
    n_rays_desired (nrd), n_rays (nr), bind_threads_(bt), bank_size_(bs),
    hpm_no (hno), hpm_string (hstr),
    c (cc), n_rays_mutex (nrm) {};
  
  shooter_thread(const shooter_thread& t) :
    execute_(false), thread_number_(t.thread_number_), x(t.x), s(t.s),
    t(t.t), rng_state(t.rng_state), n_rays_desired(t.n_rays_desired),
    n_rays(t.n_rays), bind_threads_(t.bind_threads_), bank_size_(t.bank_size_),
    hpm_no(t.hpm_no), hpm_string(t.hpm_string),
    c(t.c), n_rays_mutex(t.n_rays_mutex) {};

  void operator () ();
};


/** End-user function to shoot rays using multiple threads.  This
    very general function spawns threads and shoots, and is adaptable
    to all the cases we are interested in through the use of the
    shooter and terminator functors. The ray tracing set up is passed
    through the xfer object, which knows about the grid, emission and
    emergence objects.*/
template<typename T_xfer, typename T_shooter>  
bool
mcrx::shoot (/// xfer object contains ray-tracing parameters 
	     T_xfer& xx,
	     /// Determines which xfer::shoot function to call.
	     T_shooter s, 
	     /// Determines how the shooting can be interrupted.
	     const terminator& t,
	     /** The state vector which is used to initialize the
                 random number generators in the threads.  Must
                 contain n_threads entries. */
	     std::vector<T_rng::T_state>& rng_states,
	     /// The desired number of rays to be shot.
	     long n_rays_desired,
	     /** Reference to a total ray counter which is incremented
                 for each ray until it equals n_rays_desired (unless
                 shooting is interrupted by the terminator). This
                 counter can be started from nonzero if a run is being
                 restarted. */
	     long& n_rays,
	     /// Desired number of threads.
	     int n_threads,
	     /// If true, the threads are bound to CPUs
	     bool bind_threads,
	     /// The size of the banks that determine hyperthreading
	     /// vs real cores.
	     int bank_size,
	     /// If hpm is used, the number used in the call to hpmTstart.
	     int hpm_no,
	     /// If hpm is used, the "title" of the instrumented section.
	     std::string hpm_string)
{
  std::cout << "Spawning " << n_threads << " shooting threads" << std::endl;

  assert (rng_states.size()== n_threads);
  
  // create thread objects and
  // spawn threads
  counter c (1000);
  c = n_rays;
  boost::thread_group threads;
  typedef shooter_thread<T_xfer, T_shooter> T_thread;   
  std::vector<boost::shared_ptr<T_thread> > thread_objects;
  boost::mutex n_rays_mutex;
  for (int i = 0; i < n_threads; ++i) {
    thread_objects.push_back(boost::shared_ptr<T_thread> 
			     (new T_thread
			      (i, xx, s, t,
			       rng_states [i],
			       (n_rays_desired-n_rays)/n_threads+1,
			       n_rays, bind_threads, bank_size,
			       hpm_no, hpm_string,
			       c, n_rays_mutex)) );
  }
  if (n_threads > 1) {
    for (int i = 0; i < n_threads; ++i) {
      threads.create_thread(boost::ref(*thread_objects[i]));
    }
    // wait for them to die
    threads.join_all();
  }
  else
    // if we only have one thread, we can just execute
    (*thread_objects.front()) ();

  std::cout << "Shooting complete" << std::endl;

  // Here we strictly can't know whether they were terminated or if
  // terminator was set during the joining... I guess it doesn't
  // matter much, except that it will dump instead of save, but then
  // again that's not necessarily so bad since we want to be quick
  // about it...
  return t();
}


/** Shoots rays for the thread object. Started by the Boost threads
    module when the thread is created. Attempts to bind the thread to
    a core.  */
template<typename T_xfer, typename T_shooter>
void mcrx::shooter_thread<T_xfer, T_shooter>::operator () ()
{
  // trouble here. when the local copy goes out of scope, its copy of
  // the xfer will copy its copy of the emergence to the main
  // emergence object. But then we are left with the xfer object in
  // the first therad object, which has not received any rays and
  // which will attempt to copy its data to the emergence object too,
  // when the thread object is deleted. Also, the copy from local_copy
  // to the main emergence is subject to a race condition, since it
  // happens here, before the threads have exited. For that reason, we
  // don't make a local copy if the xfer object uses a local
  // emergence.

  if(!x.uses_local_emergence()) {
    // See if this is the local copy or not.
    if(!execute_) {
      // first thing is to bind threads and allocations so the xfer
      // copy gets allocated in the correct place
      if(bind_threads_) {
	bind_thread(thread_number_, bank_size_);
	bind_allocations();
      }

      shooter_thread<T_xfer, T_shooter> local_copy(*this);
      local_copy.execute_=true;
      local_copy();
      return;
    }
  }

  // Ensure that each thread has a distinct generator by using the
  // thread_number.
  x.rng().make_unique(thread_number_);
  x.rng().get_generator().setState(rng_state);
  DEBUG(3,boost::mutex::scoped_lock n_rays_lock (n_rays_mutex);std::cout << "Shooting thread " << thread_number_ << " seed " << x.rng().get_generator().getStateString() << std::endl;);

  // We have defined 8 stages in the xfer file. (These should be documented.)
  const int n_stages=11;
  hpm::thread_init(thread_number_, n_stages);
  hpm::thread_start(0);

  try {
    // Hardcode doing n/n_threads rays to avoid nondeterminicity. If
    // we are interrupted there will be nondeterminicity anyway
    // because we don't save the number of rays that the individual
    // threads have completed, but it's no worse than before.
    int n=0;
    for(;(n<n_rays_desired) && !t();++n)
      s (x); // Calls the appropriate xfer::shoot function
    {
      boost::mutex::scoped_lock n_rays_lock (n_rays_mutex);
      n_rays += n;
    }
  }
  catch (...) {
    std::cerr << "Whoops: Uncaught exception in thread!"  << std::endl;
    throw;
  }

  hpm::thread_stop(hpm_string.c_str());

  // return random number generator state
  rng_state = x.rng().get_generator().getState();
}

#endif
